{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "[95 97  0]\n",
      "[ 0 95 97  1  1  1  1  1  1  1]\n",
      "{'UNK': 0, 'PAD': 1, 'A': 2, 'Even': 3, 'Forrest': 4, 'Frederic': 5, 'Future': 6, 'Kirkland': 7, 'On': 8, 'Sally': 9, 'Shakespeare': 10, 'Starts': 11, 'Story': 12, 'The': 13, 'Unfortunately': 14, 'Vilmos': 15, 'WHOLE': 16, 'Zsigmond': 17, 'a': 18, 'absurd': 19, 'an': 20, 'and': 21, 'audience': 22, 'be': 23, 'better': 24, 'briefly': 25, 'by': 26, 'can': 27, 'chantings': 28, 'cinematography': 29, 'comedy': 30, 'crazy': 31, 'cryptic': 32, 'dialogue': 33, 'easy': 34, 'era': 35, 'eventually': 36, 'example': 37, 'feelings': 38, 'for': 39, 'formal': 40, 'from': 41, 'future': 42, 'general': 43, 'good': 44, 'grader': 45, 'great': 46, 'has': 47, 'insane': 48, 'into': 49, 'is': 50, 'it': 51, \"it's\": 52, 'just': 53, 'level': 54, 'make': 55, 'making': 56, 'man': 57, 'might': 58, 'mob': 59, 'narrative': 60, 'no': 61, 'of': 62, 'off': 63, 'opening': 64, 'orchestra': 65, 'out': 66, 'pig': 67, 'putting': 68, 'scene': 69, 'seem': 70, 'seen': 71, 'should': 72, 'singers': 73, 'some': 74, 'stars': 75, 'stays': 76, 'technical': 77, 'terrific': 78, 'than': 79, 'that': 80, 'the': 81, 'think': 82, 'third': 83, 'those': 84, 'time': 85, 'to': 86, 'too': 87, 'turned': 88, 'unnatural': 89, 'violent': 90, 'who': 91, 'with': 92, 'would': 93, 'you': 94, '你': 95, '哦': 96, '好': 97}\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "class Word2Sequence():\n",
    "    # unknown word 未知的单词，表示在测试集中 新单词\n",
    "    UNK_TAG = \"UNK\"\n",
    "    # padding ，填充的意思，填充的单词\n",
    "    PAD_TAG = \"PAD\"\n",
    "\n",
    "    UNK = 0\n",
    "    PAD = 1\n",
    "\n",
    "    def __init__(self):\n",
    "        self.dict = {\n",
    "            self.UNK_TAG: self.UNK,\n",
    "            self.PAD_TAG: self.PAD\n",
    "        }\n",
    "        self.fited = False\n",
    "\n",
    "    def to_index(self, word):\n",
    "        \"\"\"\n",
    "        单词到数字\n",
    "        :param word:\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        assert self.fited == True, \"必须先进行fit操作\"\n",
    "        return self.dict.get(word, self.UNK)\n",
    "\n",
    "    def to_word(self, index):\n",
    "        \"\"\"\n",
    "        数字到单词\n",
    "        :param index:\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        assert self.fited, \"必须先进行fit操作\"\n",
    "        if index in self.inversed_dict:\n",
    "            return self.inversed_dict[index]\n",
    "        return self.UNK_TAG\n",
    "\n",
    "    def __len__(self):\n",
    "        return self(self.dict)\n",
    "\n",
    "    def fit(self, sentences, min_count=1, max_count=None, max_feature=None):\n",
    "        \"\"\"\n",
    "        把单个单词保存到dict中\n",
    "        :param sentences:[[word1,word2,word3],[word1,word3,wordn..],...]\n",
    "        :param min_count: 最小出现的次数\n",
    "        :param max_count: 最大出现的次数\n",
    "        :param max_feature: 总词语的最大数量\n",
    "        :return:\n",
    "        \"\"\"\n",
    "\n",
    "        # count 表示的是单词的词频\n",
    "        count = {}\n",
    "        for sentence in sentences:\n",
    "            for a in sentence:\n",
    "                if a not in count:\n",
    "                    count[a] = 0\n",
    "                count[a] += 1\n",
    "\n",
    "        # 比最小的数量大和比最大的数量小的需要\n",
    "        if min_count is not None:\n",
    "            count = {k: v for k, v in count.items() if v >= min_count}\n",
    "        if max_count is not None:\n",
    "            count = {k: v for k, v in count.items() if v <= max_count}\n",
    "\n",
    "        # 限制最大的数量\n",
    "        if isinstance(max_feature, int):\n",
    "            count = sorted(list(count.items()), key=lambda x: x[1])\n",
    "            if max_feature is not None and len(count) > max_feature:\n",
    "                count = count[-int(max_feature):]\n",
    "            for w, _ in count:\n",
    "                self.dict[w] = len(self.dict)\n",
    "        else:\n",
    "            for w in sorted(count.keys()):\n",
    "                self.dict[w] = len(self.dict)\n",
    "\n",
    "        self.fited = True\n",
    "        # 准备一个index->word的字典\n",
    "        self.inversed_dict = dict(zip(self.dict.values(), self.dict.keys()))\n",
    "\n",
    "    def transform(self, sentence, max_len=None):\n",
    "        \"\"\"\n",
    "        实现吧句子转化为数组（向量）\n",
    "        :param sentence:\n",
    "        :param max_len:\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        assert self.fited, \"必须先进行fit操作\"\n",
    "\n",
    "        #\n",
    "        if max_len is not None:\n",
    "            r = [self.PAD] * max_len\n",
    "        else:\n",
    "            r = [self.PAD] * len(sentence)\n",
    "\n",
    "        # 如果句子的长度比 max_len长，对句子进行裁剪\n",
    "        if max_len is not None and len(sentence) > max_len:\n",
    "            sentence = sentence[:max_len]\n",
    "\n",
    "        for index, word in enumerate(sentence):\n",
    "            r[index] = self.to_index(word)\n",
    "        return np.array(r, dtype=np.int64)\n",
    "\n",
    "    def inverse_transform(self, indices):\n",
    "        \"\"\"\n",
    "        实现从数组 转化为文字\n",
    "        :param indices: [1,2,3....]\n",
    "        :return:[word1,word2.....]\n",
    "        \"\"\"\n",
    "        sentence = []\n",
    "        for i in indices:\n",
    "            word = self.to_word(i)\n",
    "            sentence.append(word)\n",
    "        return sentence\n",
    "\n",
    "\n",
    "w2s = Word2Sequence()\n",
    "# fit是传入分词后的单词\n",
    "w2s.fit([\n",
    "  ['Story', 'of', 'a', 'man', 'who', 'has', 'unnatural', 'feelings', 'for', 'a', 'pig', 'Starts', 'out', 'with', 'a', 'opening', 'scene', 'that', 'is', 'a', 'terrific', 'example', 'of', 'absurd', 'comedy', 'A', 'formal', 'orchestra', 'audience', 'is', 'turned', 'into', 'an', 'insane', 'violent', 'mob', 'by', 'the', 'crazy', 'chantings', 'of', \"it's\", 'singers', 'Unfortunately', 'it', 'stays', 'absurd', 'the', 'WHOLE', 'time', 'with', 'no', 'general', 'narrative', 'eventually', 'making', 'it', 'just', 'too', 'off', 'putting', 'Even', 'those', 'from', 'the', 'era', 'should', 'be', 'turned', 'off', 'The', 'cryptic', 'dialogue', 'would', 'make', 'Shakespeare', 'seem', 'easy', 'to', 'a', 'third', 'grader', 'On', 'a', 'technical', 'level', \"it's\", 'better', 'than', 'you', 'might', 'think', 'with', 'some', 'good', 'cinematography', 'by', 'future', 'great', 'Vilmos', 'Zsigmond', 'Future', 'stars', 'Sally', 'Kirkland', 'and', 'Frederic', 'Forrest', 'can', 'be', 'seen', 'briefly'],\n",
    "    [\"你\", \"好\", \"哦\"]])\n",
    "\n",
    "print(w2s.fited)\n",
    "print(w2s.transform([\"你\", \"好\", \"嘛\"]))\n",
    "print(w2s.transform([\"你好嘛\", \"你\", \"好\"], max_len=10))\n",
    "# [0 3 5 1 1 1 1 1 1 1]\n",
    "# 第一个单词 你好嘛，因为不存在，所有直接0\n",
    "# 第二个单词 你,因为存在所以填写3\n",
    "print(w2s.dict)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# 保存"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}